import logging
from abc import abstractmethod
from typing import Any, List, Optional

import spacy

from pydantic import BaseModel

logger = logging.getLogger(__name__)
spacy_zh = spacy.load("zh_core_web_sm")

class BaseTextTokenizer(BaseModel):
    @abstractmethod
    def tokenize_text(self, text: str) -> List[str]:
        pass


class SPACYTextTokenizer(BaseTextTokenizer):


    def tokenize_text(self, text: str) -> List[str]:  
        return [tok.text for tok in spacy_zh.tokenizer(text)]